# 파이썬 ≥3.5 필수
import sys
assert sys.version_info >= (3, 5)
# 사이킷런 ≥0.20 필수
import sklearn
assert sklearn.__version__ >= "0.20"
try:
# %tensorflow_version은 코랩에서만 동작합니다.
%tensorflow_version 2.x
IS_COLAB = True
except Exception:
IS_COLAB = False
# 텐서플로 ≥2.0 필수
import tensorflow as tf
from tensorflow import keras
assert tf.__version__ >= "2.0"
if not tf.config.list_physical_devices('GPU'):
print("감지된 GPU가 없습니다. GPU가 없으면 LSTM과 CNN이 매우 느릴 수 있습니다.")
if IS_COLAB:
print("런타임 > 런타임 유형 변경 메뉴를 선택하고 하드웨어 가속기로 GPU를 고르세요.")
# 공통 모듈 임포트
import numpy as np
import os
# 노트북 실행 결과를 동일하게 유지하기 위해
np.random.seed(42)
tf.random.set_seed(42)
# 깔끔한 그래프 출력을 위해
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)
# 그림을 저장할 위치
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "autoencoders"
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID)
os.makedirs(IMAGES_PATH, exist_ok=True)
def save_fig(fig_id, tight_layout=True, fig_extension="png", resolution=300):
path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
print("그림 저장", fig_id)
if tight_layout:
plt.tight_layout()
plt.savefig(path, format=fig_extension, dpi=resolution)17_autoencoders_and_gans
Sklearn
Python
17장 – 오토인코더와 GAN
이 노트북은 17장에 있는 모든 샘플 코드를 담고 있습니다.
 구글 코랩에서 실행하기
|
설정
먼저 몇 개의 모듈을 임포트합니다. 맷플롯립 그래프를 인라인으로 출력하도록 만들고 그림을 저장하는 함수를 준비합니다. 또한 파이썬 버전이 3.5 이상인지 확인합니다(파이썬 2.x에서도 동작하지만 곧 지원이 중단되므로 파이썬 3을 사용하는 것이 좋습니다). 사이킷런 버전이 0.20 이상인지와 텐서플로 버전이 2.0 이상인지 확인합니다.
28x28 흑백 이미지를 그리기 위한 유틸리티 함수:
def plot_image(image):
plt.imshow(image, cmap="binary")
plt.axis("off")선형 오토인코더로 PCA 수행하기
3D 데이터셋을 만듭니다:
np.random.seed(4)
def generate_3d_data(m, w1=0.1, w2=0.3, noise=0.1):
angles = np.random.rand(m) * 3 * np.pi / 2 - 0.5
data = np.empty((m, 3))
data[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * np.random.randn(m) / 2
data[:, 1] = np.sin(angles) * 0.7 + noise * np.random.randn(m) / 2
data[:, 2] = data[:, 0] * w1 + data[:, 1] * w2 + noise * np.random.randn(m)
return data
X_train = generate_3d_data(60)
X_train = X_train - X_train.mean(axis=0, keepdims=0)오토인코더를 만듭니다.
np.random.seed(42)
tf.random.set_seed(42)
encoder = keras.models.Sequential([keras.layers.Dense(2, input_shape=[3])])
decoder = keras.models.Sequential([keras.layers.Dense(3, input_shape=[2])])
autoencoder = keras.models.Sequential([encoder, decoder])
autoencoder.compile(loss="mse", optimizer=keras.optimizers.SGD(learning_rate=1.5))/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")history = autoencoder.fit(X_train, X_train, epochs=20)Epoch 1/20
2/2 [==============================] - 2s 7ms/step - loss: 0.2547
Epoch 2/20
2/2 [==============================] - 0s 5ms/step - loss: 0.1032
Epoch 3/20
2/2 [==============================] - 0s 6ms/step - loss: 0.0551
Epoch 4/20
2/2 [==============================] - 0s 4ms/step - loss: 0.0503
Epoch 5/20
2/2 [==============================] - 0s 6ms/step - loss: 0.0839
Epoch 6/20
2/2 [==============================] - 0s 5ms/step - loss: 0.2223
Epoch 7/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0913
Epoch 8/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0320
Epoch 9/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0242
Epoch 10/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0189
Epoch 11/20
2/2 [==============================] - 0s 7ms/step - loss: 0.0142
Epoch 12/20
2/2 [==============================] - 0s 3ms/step - loss: 0.0102
Epoch 13/20
2/2 [==============================] - 0s 6ms/step - loss: 0.0066
Epoch 14/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0056
Epoch 15/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0057
Epoch 16/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0053
Epoch 17/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0050
Epoch 18/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0048
Epoch 19/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0049
Epoch 20/20
2/2 [==============================] - 0s 5ms/step - loss: 0.0047codings = encoder.predict(X_train)fig = plt.figure(figsize=(4,3))
plt.plot(codings[:,0], codings[:, 1], "b.")
plt.xlabel("$z_1$", fontsize=18)
plt.ylabel("$z_2$", fontsize=18, rotation=0)
plt.grid(True)
save_fig("linear_autoencoder_pca_plot")
plt.show()Saving figure linear_autoencoder_pca_plot
적층 오토인코더
MNIST 데이터셋을 사용합니다:
(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()
X_train_full = X_train_full.astype(np.float32) / 255
X_test = X_test.astype(np.float32) / 255
X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]
y_train, y_valid = y_train_full[:-5000], y_train_full[-5000:]Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz
32768/29515 [=================================] - 0s 0us/step
40960/29515 [=========================================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz
26427392/26421880 [==============================] - 0s 0us/step
26435584/26421880 [==============================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz
16384/5148 [===============================================================================================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz
4423680/4422102 [==============================] - 0s 0us/step
4431872/4422102 [==============================] - 0s 0us/step한 번에 모든 층을 훈련하기
3개의 은닉층과 1개의 출력층(즉, 두 개를 적층)을 가진 적층 오토인코더를 만들어 보겠습니다.
def rounded_accuracy(y_true, y_pred):
return keras.metrics.binary_accuracy(tf.round(y_true), tf.round(y_pred))tf.random.set_seed(42)
np.random.seed(42)
stacked_encoder = keras.models.Sequential([
keras.layers.Flatten(input_shape=[28, 28]),
keras.layers.Dense(100, activation="selu"),
keras.layers.Dense(30, activation="selu"),
])
stacked_decoder = keras.models.Sequential([
keras.layers.Dense(100, activation="selu", input_shape=[30]),
keras.layers.Dense(28 * 28, activation="sigmoid"),
keras.layers.Reshape([28, 28])
])
stacked_ae = keras.models.Sequential([stacked_encoder, stacked_decoder])
stacked_ae.compile(loss="binary_crossentropy",
optimizer=keras.optimizers.SGD(learning_rate=1.5), metrics=[rounded_accuracy])
history = stacked_ae.fit(X_train, X_train, epochs=20,
validation_data=(X_valid, X_valid))/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")Epoch 1/20
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3381 - rounded_accuracy: 0.8870 - val_loss: 0.3173 - val_rounded_accuracy: 0.8989
Epoch 2/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.3056 - rounded_accuracy: 0.9151 - val_loss: 0.3020 - val_rounded_accuracy: 0.9199
Epoch 3/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2986 - rounded_accuracy: 0.9215 - val_loss: 0.2987 - val_rounded_accuracy: 0.9196
Epoch 4/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2948 - rounded_accuracy: 0.9249 - val_loss: 0.2937 - val_rounded_accuracy: 0.9285
Epoch 5/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2923 - rounded_accuracy: 0.9272 - val_loss: 0.2919 - val_rounded_accuracy: 0.9284
Epoch 6/20
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2904 - rounded_accuracy: 0.9289 - val_loss: 0.2915 - val_rounded_accuracy: 0.9306
Epoch 7/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2890 - rounded_accuracy: 0.9301 - val_loss: 0.2906 - val_rounded_accuracy: 0.9281
Epoch 8/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2877 - rounded_accuracy: 0.9313 - val_loss: 0.2946 - val_rounded_accuracy: 0.9296
Epoch 9/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2869 - rounded_accuracy: 0.9321 - val_loss: 0.2907 - val_rounded_accuracy: 0.9257
Epoch 10/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2859 - rounded_accuracy: 0.9330 - val_loss: 0.2878 - val_rounded_accuracy: 0.9309
Epoch 11/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2851 - rounded_accuracy: 0.9336 - val_loss: 0.2864 - val_rounded_accuracy: 0.9348
Epoch 12/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2845 - rounded_accuracy: 0.9341 - val_loss: 0.2856 - val_rounded_accuracy: 0.9355
Epoch 13/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2839 - rounded_accuracy: 0.9346 - val_loss: 0.2846 - val_rounded_accuracy: 0.9353
Epoch 14/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2835 - rounded_accuracy: 0.9350 - val_loss: 0.2844 - val_rounded_accuracy: 0.9365
Epoch 15/20
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2830 - rounded_accuracy: 0.9355 - val_loss: 0.2841 - val_rounded_accuracy: 0.9353
Epoch 16/20
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2826 - rounded_accuracy: 0.9358 - val_loss: 0.2846 - val_rounded_accuracy: 0.9355
Epoch 17/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2822 - rounded_accuracy: 0.9361 - val_loss: 0.2835 - val_rounded_accuracy: 0.9363
Epoch 18/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2819 - rounded_accuracy: 0.9363 - val_loss: 0.2831 - val_rounded_accuracy: 0.9363
Epoch 19/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2816 - rounded_accuracy: 0.9365 - val_loss: 0.2827 - val_rounded_accuracy: 0.9374
Epoch 20/20
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2813 - rounded_accuracy: 0.9368 - val_loss: 0.2840 - val_rounded_accuracy: 0.9367이 함수는 오토인코더를 사용해 몇 개의 테스트 이미지를 처리합니다. 그런 다음 원본 이미지와 재구성 이미지를 그립니다:
def show_reconstructions(model, images=X_valid, n_images=5):
reconstructions = model.predict(images[:n_images])
fig = plt.figure(figsize=(n_images * 1.5, 3))
for image_index in range(n_images):
plt.subplot(2, n_images, 1 + image_index)
plot_image(images[image_index])
plt.subplot(2, n_images, 1 + n_images + image_index)
plot_image(reconstructions[image_index])show_reconstructions(stacked_ae)
save_fig("reconstruction_plot")Saving figure reconstruction_plot
패션 MNIST 시각화하기
np.random.seed(42)
from sklearn.manifold import TSNE
X_valid_compressed = stacked_encoder.predict(X_valid)
tsne = TSNE()
X_valid_2D = tsne.fit_transform(X_valid_compressed)
X_valid_2D = (X_valid_2D - X_valid_2D.min()) / (X_valid_2D.max() - X_valid_2D.min())plt.scatter(X_valid_2D[:, 0], X_valid_2D[:, 1], c=y_valid, s=10, cmap="tab10")
plt.axis("off")
plt.show()
이 그림을 조금 더 예쁘게 그려 보겠습니다:
# https://scikit-learn.org/stable/auto_examples/manifold/plot_lle_digits.html 참고
plt.figure(figsize=(10, 8))
cmap = plt.cm.tab10
plt.scatter(X_valid_2D[:, 0], X_valid_2D[:, 1], c=y_valid, s=10, cmap=cmap)
image_positions = np.array([[1., 1.]])
for index, position in enumerate(X_valid_2D):
dist = np.sum((position - image_positions) ** 2, axis=1)
if np.min(dist) > 0.02: # if far enough from other images
image_positions = np.r_[image_positions, [position]]
imagebox = mpl.offsetbox.AnnotationBbox(
mpl.offsetbox.OffsetImage(X_valid[index], cmap="binary"),
position, bboxprops={"edgecolor": cmap(y_valid[index]), "lw": 2})
plt.gca().add_artist(imagebox)
plt.axis("off")
save_fig("fashion_mnist_visualization_plot")
plt.show()Saving figure fashion_mnist_visualization_plot
가중치 묶기
인코더의 가중치를 전치(transpose)하여 디코더의 가중치로 사용하는 식으로 인코더와 디코더의 가중치를 묶는 일은 흔합니다. 이렇게 하려면 사용자 정의 층을 사용해야 합니다.
class DenseTranspose(keras.layers.Layer):
def __init__(self, dense, activation=None, **kwargs):
self.dense = dense
self.activation = keras.activations.get(activation)
super().__init__(**kwargs)
def build(self, batch_input_shape):
self.biases = self.add_weight(name="bias",
shape=[self.dense.input_shape[-1]],
initializer="zeros")
super().build(batch_input_shape)
def call(self, inputs):
z = tf.matmul(inputs, self.dense.weights[0], transpose_b=True)
return self.activation(z + self.biases)keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)
dense_1 = keras.layers.Dense(100, activation="selu")
dense_2 = keras.layers.Dense(30, activation="selu")
tied_encoder = keras.models.Sequential([
keras.layers.Flatten(input_shape=[28, 28]),
dense_1,
dense_2
])
tied_decoder = keras.models.Sequential([
DenseTranspose(dense_2, activation="selu"),
DenseTranspose(dense_1, activation="sigmoid"),
keras.layers.Reshape([28, 28])
])
tied_ae = keras.models.Sequential([tied_encoder, tied_decoder])
tied_ae.compile(loss="binary_crossentropy",
optimizer=keras.optimizers.SGD(learning_rate=1.5), metrics=[rounded_accuracy])
history = tied_ae.fit(X_train, X_train, epochs=10,
validation_data=(X_valid, X_valid))/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")Epoch 1/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3269 - rounded_accuracy: 0.8960 - val_loss: 0.3081 - val_rounded_accuracy: 0.9077
Epoch 2/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2975 - rounded_accuracy: 0.9224 - val_loss: 0.2952 - val_rounded_accuracy: 0.9284
Epoch 3/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2920 - rounded_accuracy: 0.9275 - val_loss: 0.3016 - val_rounded_accuracy: 0.9090
Epoch 4/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2889 - rounded_accuracy: 0.9302 - val_loss: 0.2880 - val_rounded_accuracy: 0.9332
Epoch 5/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2865 - rounded_accuracy: 0.9325 - val_loss: 0.2873 - val_rounded_accuracy: 0.9316
Epoch 6/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2850 - rounded_accuracy: 0.9340 - val_loss: 0.2861 - val_rounded_accuracy: 0.9353
Epoch 7/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2838 - rounded_accuracy: 0.9350 - val_loss: 0.2858 - val_rounded_accuracy: 0.9364
Epoch 8/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2830 - rounded_accuracy: 0.9358 - val_loss: 0.2837 - val_rounded_accuracy: 0.9370
Epoch 9/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2823 - rounded_accuracy: 0.9364 - val_loss: 0.2860 - val_rounded_accuracy: 0.9301
Epoch 10/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2817 - rounded_accuracy: 0.9369 - val_loss: 0.2842 - val_rounded_accuracy: 0.9330show_reconstructions(tied_ae)
plt.show()
한 번에 하나의 오토인코더 훈련하기
def train_autoencoder(n_neurons, X_train, X_valid, loss, optimizer,
n_epochs=10, output_activation=None, metrics=None):
n_inputs = X_train.shape[-1]
encoder = keras.models.Sequential([
keras.layers.Dense(n_neurons, activation="selu", input_shape=[n_inputs])
])
decoder = keras.models.Sequential([
keras.layers.Dense(n_inputs, activation=output_activation),
])
autoencoder = keras.models.Sequential([encoder, decoder])
autoencoder.compile(optimizer, loss, metrics=metrics)
autoencoder.fit(X_train, X_train, epochs=n_epochs,
validation_data=(X_valid, X_valid))
return encoder, decoder, encoder(X_train), encoder(X_valid)tf.random.set_seed(42)
np.random.seed(42)
K = keras.backend
X_train_flat = K.batch_flatten(X_train) # equivalent to .reshape(-1, 28 * 28)
X_valid_flat = K.batch_flatten(X_valid)
enc1, dec1, X_train_enc1, X_valid_enc1 = train_autoencoder(
100, X_train_flat, X_valid_flat, "binary_crossentropy",
keras.optimizers.SGD(learning_rate=1.5), output_activation="sigmoid",
metrics=[rounded_accuracy])
enc2, dec2, _, _ = train_autoencoder(
30, X_train_enc1, X_valid_enc1, "mse", keras.optimizers.SGD(learning_rate=0.05),
output_activation="selu")/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")Epoch 1/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3445 - rounded_accuracy: 0.8874 - val_loss: 0.3123 - val_rounded_accuracy: 0.9146
Epoch 2/10
1719/1719 [==============================] - 4s 2ms/step - loss: 0.3039 - rounded_accuracy: 0.9203 - val_loss: 0.3006 - val_rounded_accuracy: 0.9246
Epoch 3/10
1719/1719 [==============================] - 4s 2ms/step - loss: 0.2949 - rounded_accuracy: 0.9286 - val_loss: 0.2934 - val_rounded_accuracy: 0.9317
Epoch 4/10
1719/1719 [==============================] - 4s 2ms/step - loss: 0.2891 - rounded_accuracy: 0.9342 - val_loss: 0.2888 - val_rounded_accuracy: 0.9363
Epoch 5/10
1719/1719 [==============================] - 4s 2ms/step - loss: 0.2853 - rounded_accuracy: 0.9378 - val_loss: 0.2857 - val_rounded_accuracy: 0.9392
Epoch 6/10
1719/1719 [==============================] - 4s 2ms/step - loss: 0.2827 - rounded_accuracy: 0.9403 - val_loss: 0.2834 - val_rounded_accuracy: 0.9409
Epoch 7/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2807 - rounded_accuracy: 0.9422 - val_loss: 0.2817 - val_rounded_accuracy: 0.9427
Epoch 8/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2792 - rounded_accuracy: 0.9437 - val_loss: 0.2803 - val_rounded_accuracy: 0.9440
Epoch 9/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2779 - rounded_accuracy: 0.9449 - val_loss: 0.2792 - val_rounded_accuracy: 0.9450
Epoch 10/10
1719/1719 [==============================] - 4s 2ms/step - loss: 0.2769 - rounded_accuracy: 0.9459 - val_loss: 0.2783 - val_rounded_accuracy: 0.9462
Epoch 1/10
1719/1719 [==============================] - 3s 2ms/step - loss: 0.5618 - val_loss: 0.3441
Epoch 2/10
1719/1719 [==============================] - 3s 2ms/step - loss: 0.2613 - val_loss: 0.2370
Epoch 3/10
1719/1719 [==============================] - 3s 2ms/step - loss: 0.2254 - val_loss: 0.2174
Epoch 4/10
1719/1719 [==============================] - 3s 2ms/step - loss: 0.2110 - val_loss: 0.2060
Epoch 5/10
1719/1719 [==============================] - 3s 2ms/step - loss: 0.2035 - val_loss: 0.1973
Epoch 6/10
1719/1719 [==============================] - 3s 2ms/step - loss: 0.1988 - val_loss: 0.1978
Epoch 7/10
1719/1719 [==============================] - 3s 2ms/step - loss: 0.1971 - val_loss: 0.1995
Epoch 8/10
1719/1719 [==============================] - 3s 2ms/step - loss: 0.1955 - val_loss: 0.2004
Epoch 9/10
1719/1719 [==============================] - 3s 2ms/step - loss: 0.1949 - val_loss: 0.1933
Epoch 10/10
1719/1719 [==============================] - 3s 2ms/step - loss: 0.1939 - val_loss: 0.1952stacked_ae_1_by_1 = keras.models.Sequential([
keras.layers.Flatten(input_shape=[28, 28]),
enc1, enc2, dec2, dec1,
keras.layers.Reshape([28, 28])
])show_reconstructions(stacked_ae_1_by_1)
plt.show()
stacked_ae_1_by_1.compile(loss="binary_crossentropy",
optimizer=keras.optimizers.SGD(learning_rate=0.1), metrics=[rounded_accuracy])
history = stacked_ae_1_by_1.fit(X_train, X_train, epochs=10,
validation_data=(X_valid, X_valid))/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")Epoch 1/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2867 - rounded_accuracy: 0.9343 - val_loss: 0.2883 - val_rounded_accuracy: 0.9341
Epoch 2/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2863 - rounded_accuracy: 0.9347 - val_loss: 0.2881 - val_rounded_accuracy: 0.9347
Epoch 3/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2861 - rounded_accuracy: 0.9349 - val_loss: 0.2879 - val_rounded_accuracy: 0.9347
Epoch 4/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2859 - rounded_accuracy: 0.9351 - val_loss: 0.2877 - val_rounded_accuracy: 0.9350
Epoch 5/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2858 - rounded_accuracy: 0.9353 - val_loss: 0.2876 - val_rounded_accuracy: 0.9351
Epoch 6/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2857 - rounded_accuracy: 0.9354 - val_loss: 0.2874 - val_rounded_accuracy: 0.9351
Epoch 7/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2856 - rounded_accuracy: 0.9355 - val_loss: 0.2874 - val_rounded_accuracy: 0.9352
Epoch 8/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2854 - rounded_accuracy: 0.9356 - val_loss: 0.2873 - val_rounded_accuracy: 0.9353
Epoch 9/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2854 - rounded_accuracy: 0.9357 - val_loss: 0.2871 - val_rounded_accuracy: 0.9353
Epoch 10/10
1719/1719 [==============================] - 4s 3ms/step - loss: 0.2853 - rounded_accuracy: 0.9358 - val_loss: 0.2871 - val_rounded_accuracy: 0.9356show_reconstructions(stacked_ae_1_by_1)
plt.show()
밀집 층 대신에 합성곱 층 사용하기
3개의 은닉층과 1개의 출력층(즉, 두 개를 적층)을 가진 적층 오토인코더를 만들어 보겠습니다.
tf.random.set_seed(42)
np.random.seed(42)
conv_encoder = keras.models.Sequential([
keras.layers.Reshape([28, 28, 1], input_shape=[28, 28]),
keras.layers.Conv2D(16, kernel_size=3, padding="SAME", activation="selu"),
keras.layers.MaxPool2D(pool_size=2),
keras.layers.Conv2D(32, kernel_size=3, padding="SAME", activation="selu"),
keras.layers.MaxPool2D(pool_size=2),
keras.layers.Conv2D(64, kernel_size=3, padding="SAME", activation="selu"),
keras.layers.MaxPool2D(pool_size=2)
])
conv_decoder = keras.models.Sequential([
keras.layers.Conv2DTranspose(32, kernel_size=3, strides=2, padding="VALID", activation="selu",
input_shape=[3, 3, 64]),
keras.layers.Conv2DTranspose(16, kernel_size=3, strides=2, padding="SAME", activation="selu"),
keras.layers.Conv2DTranspose(1, kernel_size=3, strides=2, padding="SAME", activation="sigmoid"),
keras.layers.Reshape([28, 28])
])
conv_ae = keras.models.Sequential([conv_encoder, conv_decoder])
conv_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.SGD(learning_rate=1.0),
metrics=[rounded_accuracy])
history = conv_ae.fit(X_train, X_train, epochs=5,
validation_data=(X_valid, X_valid))/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")Epoch 1/5
1719/1719 [==============================] - 22s 4ms/step - loss: 0.3018 - rounded_accuracy: 0.9187 - val_loss: 0.2848 - val_rounded_accuracy: 0.9287
Epoch 2/5
1719/1719 [==============================] - 7s 4ms/step - loss: 0.2756 - rounded_accuracy: 0.9414 - val_loss: 0.2729 - val_rounded_accuracy: 0.9456
Epoch 3/5
1719/1719 [==============================] - 7s 4ms/step - loss: 0.2708 - rounded_accuracy: 0.9462 - val_loss: 0.2696 - val_rounded_accuracy: 0.9497
Epoch 4/5
1719/1719 [==============================] - 7s 4ms/step - loss: 0.2682 - rounded_accuracy: 0.9490 - val_loss: 0.2685 - val_rounded_accuracy: 0.9492
Epoch 5/5
1719/1719 [==============================] - 7s 4ms/step - loss: 0.2664 - rounded_accuracy: 0.9510 - val_loss: 0.2671 - val_rounded_accuracy: 0.9509conv_encoder.summary()
conv_decoder.summary()Model: "sequential_10"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
reshape_2 (Reshape) (None, 28, 28, 1) 0
_________________________________________________________________
conv2d (Conv2D) (None, 28, 28, 16) 160
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 14, 14, 16) 0
_________________________________________________________________
conv2d_1 (Conv2D) (None, 14, 14, 32) 4640
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 7, 7, 32) 0
_________________________________________________________________
conv2d_2 (Conv2D) (None, 7, 7, 64) 18496
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 3, 3, 64) 0
=================================================================
Total params: 23,296
Trainable params: 23,296
Non-trainable params: 0
_________________________________________________________________
Model: "sequential_11"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_transpose (Conv2DTran (None, 7, 7, 32) 18464
_________________________________________________________________
conv2d_transpose_1 (Conv2DTr (None, 14, 14, 16) 4624
_________________________________________________________________
conv2d_transpose_2 (Conv2DTr (None, 28, 28, 1) 145
_________________________________________________________________
reshape_3 (Reshape) (None, 28, 28) 0
=================================================================
Total params: 23,233
Trainable params: 23,233
Non-trainable params: 0
_________________________________________________________________show_reconstructions(conv_ae)
plt.show()WARNING:tensorflow:5 out of the last 161 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fe6d6d61170> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.
순환 오토인코더
recurrent_encoder = keras.models.Sequential([
keras.layers.LSTM(100, return_sequences=True, input_shape=[28, 28]),
keras.layers.LSTM(30)
])
recurrent_decoder = keras.models.Sequential([
keras.layers.RepeatVector(28, input_shape=[30]),
keras.layers.LSTM(100, return_sequences=True),
keras.layers.TimeDistributed(keras.layers.Dense(28, activation="sigmoid"))
])
recurrent_ae = keras.models.Sequential([recurrent_encoder, recurrent_decoder])
recurrent_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.SGD(0.1),
metrics=[rounded_accuracy])history = recurrent_ae.fit(X_train, X_train, epochs=10, validation_data=(X_valid, X_valid))Epoch 1/10
1719/1719 [==============================] - 20s 9ms/step - loss: 0.5192 - rounded_accuracy: 0.7492 - val_loss: 0.4581 - val_rounded_accuracy: 0.8081
Epoch 2/10
1719/1719 [==============================] - 15s 9ms/step - loss: 0.4050 - rounded_accuracy: 0.8432 - val_loss: 0.3743 - val_rounded_accuracy: 0.8687
Epoch 3/10
1719/1719 [==============================] - 15s 9ms/step - loss: 0.3653 - rounded_accuracy: 0.8710 - val_loss: 0.3601 - val_rounded_accuracy: 0.8766
Epoch 4/10
1719/1719 [==============================] - 15s 9ms/step - loss: 0.3507 - rounded_accuracy: 0.8809 - val_loss: 0.3522 - val_rounded_accuracy: 0.8773
Epoch 5/10
1719/1719 [==============================] - 14s 8ms/step - loss: 0.3405 - rounded_accuracy: 0.8875 - val_loss: 0.3361 - val_rounded_accuracy: 0.8917
Epoch 6/10
1719/1719 [==============================] - 14s 8ms/step - loss: 0.3335 - rounded_accuracy: 0.8922 - val_loss: 0.3305 - val_rounded_accuracy: 0.8963
Epoch 7/10
1719/1719 [==============================] - 14s 8ms/step - loss: 0.3287 - rounded_accuracy: 0.8956 - val_loss: 0.3333 - val_rounded_accuracy: 0.8931
Epoch 8/10
1719/1719 [==============================] - 14s 8ms/step - loss: 0.3249 - rounded_accuracy: 0.8982 - val_loss: 0.3237 - val_rounded_accuracy: 0.8997
Epoch 9/10
1719/1719 [==============================] - 15s 9ms/step - loss: 0.3222 - rounded_accuracy: 0.9002 - val_loss: 0.3254 - val_rounded_accuracy: 0.9004
Epoch 10/10
1719/1719 [==============================] - 14s 8ms/step - loss: 0.3198 - rounded_accuracy: 0.9021 - val_loss: 0.3237 - val_rounded_accuracy: 0.8982show_reconstructions(recurrent_ae)
plt.show()WARNING:tensorflow:6 out of the last 162 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fe70ce24d40> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.
적층 잡음 제거 오토인코더
가우시안 잡음을 사용합니다:
tf.random.set_seed(42)
np.random.seed(42)
denoising_encoder = keras.models.Sequential([
keras.layers.Flatten(input_shape=[28, 28]),
keras.layers.GaussianNoise(0.2),
keras.layers.Dense(100, activation="selu"),
keras.layers.Dense(30, activation="selu")
])
denoising_decoder = keras.models.Sequential([
keras.layers.Dense(100, activation="selu", input_shape=[30]),
keras.layers.Dense(28 * 28, activation="sigmoid"),
keras.layers.Reshape([28, 28])
])
denoising_ae = keras.models.Sequential([denoising_encoder, denoising_decoder])
denoising_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.SGD(learning_rate=1.0),
metrics=[rounded_accuracy])
history = denoising_ae.fit(X_train, X_train, epochs=10,
validation_data=(X_valid, X_valid))/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")Epoch 1/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3509 - rounded_accuracy: 0.8766 - val_loss: 0.3182 - val_rounded_accuracy: 0.9036
Epoch 2/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3121 - rounded_accuracy: 0.9097 - val_loss: 0.3086 - val_rounded_accuracy: 0.9156
Epoch 3/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3057 - rounded_accuracy: 0.9152 - val_loss: 0.3035 - val_rounded_accuracy: 0.9191
Epoch 4/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3021 - rounded_accuracy: 0.9183 - val_loss: 0.2999 - val_rounded_accuracy: 0.9207
Epoch 5/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2992 - rounded_accuracy: 0.9209 - val_loss: 0.2968 - val_rounded_accuracy: 0.9252
Epoch 6/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2969 - rounded_accuracy: 0.9229 - val_loss: 0.2947 - val_rounded_accuracy: 0.9272
Epoch 7/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2953 - rounded_accuracy: 0.9243 - val_loss: 0.2946 - val_rounded_accuracy: 0.9275
Epoch 8/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2940 - rounded_accuracy: 0.9254 - val_loss: 0.2931 - val_rounded_accuracy: 0.9296
Epoch 9/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2931 - rounded_accuracy: 0.9262 - val_loss: 0.2923 - val_rounded_accuracy: 0.9266
Epoch 10/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.2922 - rounded_accuracy: 0.9270 - val_loss: 0.2907 - val_rounded_accuracy: 0.9299tf.random.set_seed(42)
np.random.seed(42)
noise = keras.layers.GaussianNoise(0.2)
show_reconstructions(denoising_ae, noise(X_valid, training=True))
plt.show()
드롭아웃을 사용합니다:
tf.random.set_seed(42)
np.random.seed(42)
dropout_encoder = keras.models.Sequential([
keras.layers.Flatten(input_shape=[28, 28]),
keras.layers.Dropout(0.5),
keras.layers.Dense(100, activation="selu"),
keras.layers.Dense(30, activation="selu")
])
dropout_decoder = keras.models.Sequential([
keras.layers.Dense(100, activation="selu", input_shape=[30]),
keras.layers.Dense(28 * 28, activation="sigmoid"),
keras.layers.Reshape([28, 28])
])
dropout_ae = keras.models.Sequential([dropout_encoder, dropout_decoder])
dropout_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.SGD(learning_rate=1.0),
metrics=[rounded_accuracy])
history = dropout_ae.fit(X_train, X_train, epochs=10,
validation_data=(X_valid, X_valid))/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")Epoch 1/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3568 - rounded_accuracy: 0.8710 - val_loss: 0.3200 - val_rounded_accuracy: 0.9041
Epoch 2/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3182 - rounded_accuracy: 0.9032 - val_loss: 0.3125 - val_rounded_accuracy: 0.9110
Epoch 3/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3128 - rounded_accuracy: 0.9075 - val_loss: 0.3075 - val_rounded_accuracy: 0.9152
Epoch 4/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3092 - rounded_accuracy: 0.9102 - val_loss: 0.3041 - val_rounded_accuracy: 0.9178
Epoch 5/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3067 - rounded_accuracy: 0.9123 - val_loss: 0.3015 - val_rounded_accuracy: 0.9193
Epoch 6/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3048 - rounded_accuracy: 0.9139 - val_loss: 0.3014 - val_rounded_accuracy: 0.9173
Epoch 7/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3033 - rounded_accuracy: 0.9151 - val_loss: 0.2995 - val_rounded_accuracy: 0.9210
Epoch 8/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3022 - rounded_accuracy: 0.9159 - val_loss: 0.2978 - val_rounded_accuracy: 0.9229
Epoch 9/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3012 - rounded_accuracy: 0.9167 - val_loss: 0.2971 - val_rounded_accuracy: 0.9221
Epoch 10/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3003 - rounded_accuracy: 0.9175 - val_loss: 0.2958 - val_rounded_accuracy: 0.9238tf.random.set_seed(42)
np.random.seed(42)
dropout = keras.layers.Dropout(0.5)
show_reconstructions(dropout_ae, dropout(X_valid, training=True))
save_fig("dropout_denoising_plot", tight_layout=False)Saving figure dropout_denoising_plot
희소 오토인코더
간단한 적층 오토인코더를 만들어 희소 오토인코더와 비교하겠습니다. 이번에는 코딩 층에 시그모이드 활성화 함수를 사용하여 코딩 값의 범위가 0~1 사이가 되도록 만듭니다:
tf.random.set_seed(42)
np.random.seed(42)
simple_encoder = keras.models.Sequential([
keras.layers.Flatten(input_shape=[28, 28]),
keras.layers.Dense(100, activation="selu"),
keras.layers.Dense(30, activation="sigmoid"),
])
simple_decoder = keras.models.Sequential([
keras.layers.Dense(100, activation="selu", input_shape=[30]),
keras.layers.Dense(28 * 28, activation="sigmoid"),
keras.layers.Reshape([28, 28])
])
simple_ae = keras.models.Sequential([simple_encoder, simple_decoder])
simple_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.SGD(learning_rate=1.),
metrics=[rounded_accuracy])
history = simple_ae.fit(X_train, X_train, epochs=10,
validation_data=(X_valid, X_valid))/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")Epoch 1/10
1719/1719 [==============================] - 6s 3ms/step - loss: 0.4329 - rounded_accuracy: 0.7950 - val_loss: 0.3773 - val_rounded_accuracy: 0.8492
Epoch 2/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3612 - rounded_accuracy: 0.8668 - val_loss: 0.3514 - val_rounded_accuracy: 0.8797
Epoch 3/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3410 - rounded_accuracy: 0.8852 - val_loss: 0.3367 - val_rounded_accuracy: 0.8912
Epoch 4/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3288 - rounded_accuracy: 0.8954 - val_loss: 0.3263 - val_rounded_accuracy: 0.8992
Epoch 5/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3213 - rounded_accuracy: 0.9012 - val_loss: 0.3210 - val_rounded_accuracy: 0.9032
Epoch 6/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3176 - rounded_accuracy: 0.9038 - val_loss: 0.3179 - val_rounded_accuracy: 0.9050
Epoch 7/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3150 - rounded_accuracy: 0.9060 - val_loss: 0.3161 - val_rounded_accuracy: 0.9090
Epoch 8/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3129 - rounded_accuracy: 0.9079 - val_loss: 0.3154 - val_rounded_accuracy: 0.9108
Epoch 9/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3109 - rounded_accuracy: 0.9096 - val_loss: 0.3133 - val_rounded_accuracy: 0.9085
Epoch 10/10
1719/1719 [==============================] - 5s 3ms/step - loss: 0.3092 - rounded_accuracy: 0.9112 - val_loss: 0.3098 - val_rounded_accuracy: 0.9118show_reconstructions(simple_ae)
plt.show()
활성화 히스토그램을 출력하기 위한 함수를 만듭니다:
def plot_percent_hist(ax, data, bins):
counts, _ = np.histogram(data, bins=bins)
widths = bins[1:] - bins[:-1]
x = bins[:-1] + widths / 2
ax.bar(x, counts / len(data), width=widths*0.8)
ax.xaxis.set_ticks(bins)
ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(
lambda y, position: "{}%".format(int(np.round(100 * y)))))
ax.grid(True)def plot_activations_histogram(encoder, height=1, n_bins=10):
X_valid_codings = encoder(X_valid).numpy()
activation_means = X_valid_codings.mean(axis=0)
mean = activation_means.mean()
bins = np.linspace(0, 1, n_bins + 1)
fig, [ax1, ax2] = plt.subplots(figsize=(10, 3), nrows=1, ncols=2, sharey=True)
plot_percent_hist(ax1, X_valid_codings.ravel(), bins)
ax1.plot([mean, mean], [0, height], "k--", label="Overall Mean = {:.2f}".format(mean))
ax1.legend(loc="upper center", fontsize=14)
ax1.set_xlabel("Activation")
ax1.set_ylabel("% Activations")
ax1.axis([0, 1, 0, height])
plot_percent_hist(ax2, activation_means, bins)
ax2.plot([mean, mean], [0, height], "k--")
ax2.set_xlabel("Neuron Mean Activation")
ax2.set_ylabel("% Neurons")
ax2.axis([0, 1, 0, height])이 함수를 사용해 인코딩 층의 활성화에 대한 히스토그램을 출력해 보죠. 왼쪽의 히스토그램은 전체 활성화의 분포를 보여줍니다. 0과 1에 가까운 값이 전체적으로 더 많이 등장합니다. 이는 시그모이드 함수가 포화되는 특성 때문입니다. 오른쪽의 히스토그램은 평균 뉴런의 분포를 보여줍니다. 대부분의 뉴런이 0.5에 가까운 평균 활성화를 가지고 있습니다. 두 히스토그램은 각 뉴런이 50% 확률로 0이나 1에 가까운 값에 활성화된다는 것을 보여줍니다. 하지만 일부 뉴런은 거의 항상 활성화됩니다(오른쪽 히스토그램의 오른쪽편).
plot_activations_histogram(simple_encoder, height=0.35)
plt.show()
코딩 층에 \(\ell_1\) 규제를 추가해 보죠:
tf.random.set_seed(42)
np.random.seed(42)
sparse_l1_encoder = keras.models.Sequential([
keras.layers.Flatten(input_shape=[28, 28]),
keras.layers.Dense(100, activation="selu"),
keras.layers.Dense(300, activation="sigmoid"),
keras.layers.ActivityRegularization(l1=1e-3) # Alternatively, you could add
# activity_regularizer=keras.regularizers.l1(1e-3)
# to the previous layer.
])
sparse_l1_decoder = keras.models.Sequential([
keras.layers.Dense(100, activation="selu", input_shape=[300]),
keras.layers.Dense(28 * 28, activation="sigmoid"),
keras.layers.Reshape([28, 28])
])
sparse_l1_ae = keras.models.Sequential([sparse_l1_encoder, sparse_l1_decoder])
sparse_l1_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.SGD(learning_rate=1.0),
metrics=[rounded_accuracy])
history = sparse_l1_ae.fit(X_train, X_train, epochs=10,
validation_data=(X_valid, X_valid))/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")Epoch 1/10
1719/1719 [==============================] - 7s 4ms/step - loss: 0.4310 - rounded_accuracy: 0.8129 - val_loss: 0.3808 - val_rounded_accuracy: 0.8555
Epoch 2/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3690 - rounded_accuracy: 0.8689 - val_loss: 0.3638 - val_rounded_accuracy: 0.8741
Epoch 3/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3545 - rounded_accuracy: 0.8799 - val_loss: 0.3502 - val_rounded_accuracy: 0.8857
Epoch 4/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3437 - rounded_accuracy: 0.8876 - val_loss: 0.3418 - val_rounded_accuracy: 0.8898
Epoch 5/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3372 - rounded_accuracy: 0.8920 - val_loss: 0.3368 - val_rounded_accuracy: 0.8949
Epoch 6/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3320 - rounded_accuracy: 0.8968 - val_loss: 0.3316 - val_rounded_accuracy: 0.8992
Epoch 7/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3279 - rounded_accuracy: 0.9000 - val_loss: 0.3275 - val_rounded_accuracy: 0.9031
Epoch 8/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3245 - rounded_accuracy: 0.9029 - val_loss: 0.3256 - val_rounded_accuracy: 0.9060
Epoch 9/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3219 - rounded_accuracy: 0.9051 - val_loss: 0.3240 - val_rounded_accuracy: 0.9015
Epoch 10/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3201 - rounded_accuracy: 0.9065 - val_loss: 0.3210 - val_rounded_accuracy: 0.9085show_reconstructions(sparse_l1_ae)
plot_activations_histogram(sparse_l1_encoder, height=1.)
plt.show()
희소성을 위해 KL 발산을 사용하여 0% 아니라 10% 희소성을 만들어 보죠:
p = 0.1
q = np.linspace(0.001, 0.999, 500)
kl_div = p * np.log(p / q) + (1 - p) * np.log((1 - p) / (1 - q))
mse = (p - q)**2
mae = np.abs(p - q)
plt.plot([p, p], [0, 0.3], "k:")
plt.text(0.05, 0.32, "Target\nsparsity", fontsize=14)
plt.plot(q, kl_div, "b-", label="KL divergence")
plt.plot(q, mae, "g--", label=r"MAE ($\ell_1$)")
plt.plot(q, mse, "r--", linewidth=1, label=r"MSE ($\ell_2$)")
plt.legend(loc="upper left", fontsize=14)
plt.xlabel("Actual sparsity")
plt.ylabel("Cost", rotation=0)
plt.axis([0, 1, 0, 0.95])
save_fig("sparsity_loss_plot")Saving figure sparsity_loss_plot
K = keras.backend
kl_divergence = keras.losses.kullback_leibler_divergence
class KLDivergenceRegularizer(keras.regularizers.Regularizer):
def __init__(self, weight, target=0.1):
self.weight = weight
self.target = target
def __call__(self, inputs):
mean_activities = K.mean(inputs, axis=0)
return self.weight * (
kl_divergence(self.target, mean_activities) +
kl_divergence(1. - self.target, 1. - mean_activities))tf.random.set_seed(42)
np.random.seed(42)
kld_reg = KLDivergenceRegularizer(weight=0.05, target=0.1)
sparse_kl_encoder = keras.models.Sequential([
keras.layers.Flatten(input_shape=[28, 28]),
keras.layers.Dense(100, activation="selu"),
keras.layers.Dense(300, activation="sigmoid", activity_regularizer=kld_reg)
])
sparse_kl_decoder = keras.models.Sequential([
keras.layers.Dense(100, activation="selu", input_shape=[300]),
keras.layers.Dense(28 * 28, activation="sigmoid"),
keras.layers.Reshape([28, 28])
])
sparse_kl_ae = keras.models.Sequential([sparse_kl_encoder, sparse_kl_decoder])
sparse_kl_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.SGD(learning_rate=1.0),
metrics=[rounded_accuracy])
history = sparse_kl_ae.fit(X_train, X_train, epochs=10,
validation_data=(X_valid, X_valid))/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
"The `lr` argument is deprecated, use `learning_rate` instead.")Epoch 1/10
1719/1719 [==============================] - 9s 5ms/step - loss: 0.4150 - rounded_accuracy: 0.8121 - val_loss: 0.3716 - val_rounded_accuracy: 0.8564
Epoch 2/10
1719/1719 [==============================] - 8s 4ms/step - loss: 0.3531 - rounded_accuracy: 0.8763 - val_loss: 0.3442 - val_rounded_accuracy: 0.8847
Epoch 3/10
1719/1719 [==============================] - 8s 5ms/step - loss: 0.3340 - rounded_accuracy: 0.8918 - val_loss: 0.3293 - val_rounded_accuracy: 0.8975
Epoch 4/10
1719/1719 [==============================] - 8s 5ms/step - loss: 0.3224 - rounded_accuracy: 0.9018 - val_loss: 0.3213 - val_rounded_accuracy: 0.9043
Epoch 5/10
1719/1719 [==============================] - 8s 5ms/step - loss: 0.3169 - rounded_accuracy: 0.9063 - val_loss: 0.3171 - val_rounded_accuracy: 0.9078
Epoch 6/10
1719/1719 [==============================] - 9s 5ms/step - loss: 0.3135 - rounded_accuracy: 0.9093 - val_loss: 0.3140 - val_rounded_accuracy: 0.9105
Epoch 7/10
1719/1719 [==============================] - 7s 4ms/step - loss: 0.3107 - rounded_accuracy: 0.9117 - val_loss: 0.3115 - val_rounded_accuracy: 0.9130
Epoch 8/10
1719/1719 [==============================] - 7s 4ms/step - loss: 0.3084 - rounded_accuracy: 0.9137 - val_loss: 0.3092 - val_rounded_accuracy: 0.9149
Epoch 9/10
1719/1719 [==============================] - 7s 4ms/step - loss: 0.3063 - rounded_accuracy: 0.9155 - val_loss: 0.3074 - val_rounded_accuracy: 0.9145
Epoch 10/10
1719/1719 [==============================] - 7s 4ms/step - loss: 0.3043 - rounded_accuracy: 0.9171 - val_loss: 0.3054 - val_rounded_accuracy: 0.9169show_reconstructions(sparse_kl_ae)
plot_activations_histogram(sparse_kl_encoder)
save_fig("sparse_autoencoder_plot")
plt.show()Saving figure sparse_autoencoder_plot
변이형 오토인코더
class Sampling(keras.layers.Layer):
def call(self, inputs):
mean, log_var = inputs
return K.random_normal(tf.shape(log_var)) * K.exp(log_var / 2) + mean tf.random.set_seed(42)
np.random.seed(42)
codings_size = 10
inputs = keras.layers.Input(shape=[28, 28])
z = keras.layers.Flatten()(inputs)
z = keras.layers.Dense(150, activation="selu")(z)
z = keras.layers.Dense(100, activation="selu")(z)
codings_mean = keras.layers.Dense(codings_size)(z)
codings_log_var = keras.layers.Dense(codings_size)(z)
codings = Sampling()([codings_mean, codings_log_var])
variational_encoder = keras.models.Model(
inputs=[inputs], outputs=[codings_mean, codings_log_var, codings])
decoder_inputs = keras.layers.Input(shape=[codings_size])
x = keras.layers.Dense(100, activation="selu")(decoder_inputs)
x = keras.layers.Dense(150, activation="selu")(x)
x = keras.layers.Dense(28 * 28, activation="sigmoid")(x)
outputs = keras.layers.Reshape([28, 28])(x)
variational_decoder = keras.models.Model(inputs=[decoder_inputs], outputs=[outputs])
_, _, codings = variational_encoder(inputs)
reconstructions = variational_decoder(codings)
variational_ae = keras.models.Model(inputs=[inputs], outputs=[reconstructions])
latent_loss = -0.5 * K.sum(
1 + codings_log_var - K.exp(codings_log_var) - K.square(codings_mean),
axis=-1)
variational_ae.add_loss(K.mean(latent_loss) / 784.)
variational_ae.compile(loss="binary_crossentropy", optimizer="rmsprop", metrics=[rounded_accuracy])
history = variational_ae.fit(X_train, X_train, epochs=25, batch_size=128,
validation_data=(X_valid, X_valid))Epoch 1/25
430/430 [==============================] - 5s 7ms/step - loss: 0.3895 - rounded_accuracy: 0.8608 - val_loss: 0.3493 - val_rounded_accuracy: 0.8964
Epoch 2/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3426 - rounded_accuracy: 0.8979 - val_loss: 0.3416 - val_rounded_accuracy: 0.9021
Epoch 3/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3326 - rounded_accuracy: 0.9054 - val_loss: 0.3354 - val_rounded_accuracy: 0.9025
Epoch 4/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3277 - rounded_accuracy: 0.9092 - val_loss: 0.3296 - val_rounded_accuracy: 0.9062
Epoch 5/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3246 - rounded_accuracy: 0.9120 - val_loss: 0.3263 - val_rounded_accuracy: 0.9133
Epoch 6/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3224 - rounded_accuracy: 0.9137 - val_loss: 0.3242 - val_rounded_accuracy: 0.9105
Epoch 7/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3207 - rounded_accuracy: 0.9151 - val_loss: 0.3201 - val_rounded_accuracy: 0.9185
Epoch 8/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3195 - rounded_accuracy: 0.9161 - val_loss: 0.3218 - val_rounded_accuracy: 0.9171
Epoch 9/25
430/430 [==============================] - 3s 7ms/step - loss: 0.3185 - rounded_accuracy: 0.9170 - val_loss: 0.3255 - val_rounded_accuracy: 0.9072
Epoch 10/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3176 - rounded_accuracy: 0.9176 - val_loss: 0.3201 - val_rounded_accuracy: 0.9157
Epoch 11/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3170 - rounded_accuracy: 0.9183 - val_loss: 0.3187 - val_rounded_accuracy: 0.9197
Epoch 12/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3163 - rounded_accuracy: 0.9190 - val_loss: 0.3194 - val_rounded_accuracy: 0.9204
Epoch 13/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3158 - rounded_accuracy: 0.9194 - val_loss: 0.3175 - val_rounded_accuracy: 0.9203
Epoch 14/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3153 - rounded_accuracy: 0.9197 - val_loss: 0.3174 - val_rounded_accuracy: 0.9185
Epoch 15/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3149 - rounded_accuracy: 0.9199 - val_loss: 0.3176 - val_rounded_accuracy: 0.9154
Epoch 16/25
430/430 [==============================] - 3s 7ms/step - loss: 0.3145 - rounded_accuracy: 0.9203 - val_loss: 0.3189 - val_rounded_accuracy: 0.9184
Epoch 17/25
430/430 [==============================] - 3s 6ms/step - loss: 0.3141 - rounded_accuracy: 0.9207 - val_loss: 0.3182 - val_rounded_accuracy: 0.9183
Epoch 18/25
430/430 [==============================] - 2s 6ms/step - loss: 0.3138 - rounded_accuracy: 0.9208 - val_loss: 0.3157 - val_rounded_accuracy: 0.9219
Epoch 19/25
430/430 [==============================] - 2s 6ms/step - loss: 0.3135 - rounded_accuracy: 0.9211 - val_loss: 0.3157 - val_rounded_accuracy: 0.9220
Epoch 20/25
430/430 [==============================] - 2s 5ms/step - loss: 0.3132 - rounded_accuracy: 0.9213 - val_loss: 0.3162 - val_rounded_accuracy: 0.9179
Epoch 21/25
430/430 [==============================] - 2s 5ms/step - loss: 0.3129 - rounded_accuracy: 0.9215 - val_loss: 0.3165 - val_rounded_accuracy: 0.9222
Epoch 22/25
430/430 [==============================] - 2s 5ms/step - loss: 0.3127 - rounded_accuracy: 0.9216 - val_loss: 0.3139 - val_rounded_accuracy: 0.9222
Epoch 23/25
430/430 [==============================] - 2s 5ms/step - loss: 0.3125 - rounded_accuracy: 0.9216 - val_loss: 0.3177 - val_rounded_accuracy: 0.9213
Epoch 24/25
430/430 [==============================] - 2s 5ms/step - loss: 0.3123 - rounded_accuracy: 0.9219 - val_loss: 0.3136 - val_rounded_accuracy: 0.9213
Epoch 25/25
430/430 [==============================] - 2s 5ms/step - loss: 0.3121 - rounded_accuracy: 0.9221 - val_loss: 0.3156 - val_rounded_accuracy: 0.9218show_reconstructions(variational_ae)
plt.show()
패션 이미지 생성하기
def plot_multiple_images(images, n_cols=None):
n_cols = n_cols or len(images)
n_rows = (len(images) - 1) // n_cols + 1
if images.shape[-1] == 1:
images = np.squeeze(images, axis=-1)
plt.figure(figsize=(n_cols, n_rows))
for index, image in enumerate(images):
plt.subplot(n_rows, n_cols, index + 1)
plt.imshow(image, cmap="binary")
plt.axis("off")몇 개의 랜덤한 코딩을 생성하고 이를 디코딩하여 결과 이미지를 출력합니다:
tf.random.set_seed(42)
codings = tf.random.normal(shape=[12, codings_size])
images = variational_decoder(codings).numpy()
plot_multiple_images(images, 4)
save_fig("vae_generated_images_plot", tight_layout=False)Saving figure vae_generated_images_plot
이미지 사이에서 의미 보간을 수행해 보죠:
tf.random.set_seed(42)
np.random.seed(42)
codings_grid = tf.reshape(codings, [1, 3, 4, codings_size])
larger_grid = tf.image.resize(codings_grid, size=[5, 7])
interpolated_codings = tf.reshape(larger_grid, [-1, codings_size])
images = variational_decoder(interpolated_codings).numpy()
plt.figure(figsize=(7, 5))
for index, image in enumerate(images):
plt.subplot(5, 7, index + 1)
if index%7%2==0 and index//7%2==0:
plt.gca().get_xaxis().set_visible(False)
plt.gca().get_yaxis().set_visible(False)
else:
plt.axis("off")
plt.imshow(image, cmap="binary")
save_fig("semantic_interpolation_plot", tight_layout=False)Saving figure semantic_interpolation_plot
생성적 적대 신경망
np.random.seed(42)
tf.random.set_seed(42)
codings_size = 30
generator = keras.models.Sequential([
keras.layers.Dense(100, activation="selu", input_shape=[codings_size]),
keras.layers.Dense(150, activation="selu"),
keras.layers.Dense(28 * 28, activation="sigmoid"),
keras.layers.Reshape([28, 28])
])
discriminator = keras.models.Sequential([
keras.layers.Flatten(input_shape=[28, 28]),
keras.layers.Dense(150, activation="selu"),
keras.layers.Dense(100, activation="selu"),
keras.layers.Dense(1, activation="sigmoid")
])
gan = keras.models.Sequential([generator, discriminator])discriminator.compile(loss="binary_crossentropy", optimizer="rmsprop")
discriminator.trainable = False
gan.compile(loss="binary_crossentropy", optimizer="rmsprop")batch_size = 32
dataset = tf.data.Dataset.from_tensor_slices(X_train).shuffle(1000)
dataset = dataset.batch(batch_size, drop_remainder=True).prefetch(1)def train_gan(gan, dataset, batch_size, codings_size, n_epochs=50):
generator, discriminator = gan.layers
for epoch in range(n_epochs):
print("Epoch {}/{}".format(epoch + 1, n_epochs)) # not shown in the book
for X_batch in dataset:
# phase 1 - training the discriminator
noise = tf.random.normal(shape=[batch_size, codings_size])
generated_images = generator(noise)
X_fake_and_real = tf.concat([generated_images, X_batch], axis=0)
y1 = tf.constant([[0.]] * batch_size + [[1.]] * batch_size)
discriminator.trainable = True
discriminator.train_on_batch(X_fake_and_real, y1)
# phase 2 - training the generator
noise = tf.random.normal(shape=[batch_size, codings_size])
y2 = tf.constant([[1.]] * batch_size)
discriminator.trainable = False
gan.train_on_batch(noise, y2)
plot_multiple_images(generated_images, 8) # not shown
plt.show() # not showntrain_gan(gan, dataset, batch_size, codings_size, n_epochs=1)Epoch 1/1
tf.random.set_seed(42)
np.random.seed(42)
noise = tf.random.normal(shape=[batch_size, codings_size])
generated_images = generator(noise)
plot_multiple_images(generated_images, 8)
save_fig("gan_generated_images_plot", tight_layout=False)Saving figure gan_generated_images_plot
train_gan(gan, dataset, batch_size, codings_size)Epoch 1/50
Epoch 2/50
Epoch 3/50
Epoch 4/50
Epoch 5/50
Epoch 6/50
Epoch 7/50
Epoch 8/50
Epoch 9/50
Epoch 10/50
Epoch 11/50
Epoch 12/50
Epoch 13/50
Epoch 14/50
Epoch 15/50
Epoch 16/50
Epoch 17/50
Epoch 18/50
Epoch 19/50
Epoch 20/50
Epoch 21/50
Epoch 22/50
Epoch 23/50
Epoch 24/50
Epoch 25/50
Epoch 26/50
Epoch 27/50
Epoch 28/50
Epoch 29/50
Epoch 30/50
Epoch 31/50
Epoch 32/50
Epoch 33/50
Epoch 34/50
Epoch 35/50
Epoch 36/50
Epoch 37/50
Epoch 38/50
Epoch 39/50
Epoch 40/50
Epoch 41/50
Epoch 42/50
Epoch 43/50
Epoch 44/50
Epoch 45/50
Epoch 46/50
Epoch 47/50
Epoch 48/50
Epoch 49/50
Epoch 50/50
심층 합성곱 GAN
tf.random.set_seed(42)
np.random.seed(42)
codings_size = 100
generator = keras.models.Sequential([
keras.layers.Dense(7 * 7 * 128, input_shape=[codings_size]),
keras.layers.Reshape([7, 7, 128]),
keras.layers.BatchNormalization(),
keras.layers.Conv2DTranspose(64, kernel_size=5, strides=2, padding="SAME",
activation="selu"),
keras.layers.BatchNormalization(),
keras.layers.Conv2DTranspose(1, kernel_size=5, strides=2, padding="SAME",
activation="tanh"),
])
discriminator = keras.models.Sequential([
keras.layers.Conv2D(64, kernel_size=5, strides=2, padding="SAME",
activation=keras.layers.LeakyReLU(0.2),
input_shape=[28, 28, 1]),
keras.layers.Dropout(0.4),
keras.layers.Conv2D(128, kernel_size=5, strides=2, padding="SAME",
activation=keras.layers.LeakyReLU(0.2)),
keras.layers.Dropout(0.4),
keras.layers.Flatten(),
keras.layers.Dense(1, activation="sigmoid")
])
gan = keras.models.Sequential([generator, discriminator])discriminator.compile(loss="binary_crossentropy", optimizer="rmsprop")
discriminator.trainable = False
gan.compile(loss="binary_crossentropy", optimizer="rmsprop")X_train_dcgan = X_train.reshape(-1, 28, 28, 1) * 2. - 1. # reshape and rescalebatch_size = 32
dataset = tf.data.Dataset.from_tensor_slices(X_train_dcgan)
dataset = dataset.shuffle(1000)
dataset = dataset.batch(batch_size, drop_remainder=True).prefetch(1)train_gan(gan, dataset, batch_size, codings_size)Epoch 1/50
Epoch 2/50
Epoch 3/50
Epoch 4/50
Epoch 5/50
Epoch 6/50
Epoch 7/50
Epoch 8/50
Epoch 9/50
Epoch 10/50
Epoch 11/50
Epoch 12/50
Epoch 13/50
Epoch 14/50
Epoch 15/50
Epoch 16/50
Epoch 17/50
Epoch 18/50
Epoch 19/50
Epoch 20/50
Epoch 21/50
Epoch 22/50
Epoch 23/50
Epoch 24/50
Epoch 25/50
Epoch 26/50
Epoch 27/50
Epoch 28/50
Epoch 29/50
Epoch 30/50
Epoch 31/50
Epoch 32/50
Epoch 33/50
Epoch 34/50
Epoch 35/50
Epoch 36/50
Epoch 37/50
Epoch 38/50
Epoch 39/50
Epoch 40/50
Epoch 41/50
Epoch 42/50
Epoch 43/50
Epoch 44/50
Epoch 45/50
Epoch 46/50
Epoch 47/50
Epoch 48/50
Epoch 49/50
Epoch 50/50
tf.random.set_seed(42)
np.random.seed(42)
noise = tf.random.normal(shape=[batch_size, codings_size])
generated_images = generator(noise)
plot_multiple_images(generated_images, 8)
save_fig("dcgan_generated_images_plot", tight_layout=False)Saving figure dcgan_generated_images_plot
추가 자료
이진 오토인코더를 사용한 해싱
패션 MNIST 데이터셋을 다시 로드합니다:
(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()
X_train_full = X_train_full.astype(np.float32) / 255
X_test = X_test.astype(np.float32) / 255
X_train, X_valid = X_train_full[:-5000], X_train_full[-5000:]
y_train, y_valid = y_train_full[:-5000], y_train_full[-5000:]인코더의 출력 층이 시그모이드 활성화 함수와 16개의 뉴런을 가지고 있고 가우시안 잡음이 그 앞에 놓인 오토인코더를 훈련해 보죠. 훈련하는 동안 잡음 층이 이전 층이 큰 값을 출력하도록 학습시킵니다. 작은 값은 잡음에 묻히기 때문입니다. 결국 시그모이드 활성화 함수 덕분에 출력층은 0~1 사이의 값을 출력합니다. 출력값을 0과 1로 반올림하면 16비트 “시맨틱” 해시를 얻습니다. 모든 것이 잘 수행되면 비슷한 이미지는 같은 해시를 가지게 될 것입니다. 이는 검색 엔진에 매우 유용합니다. 예를 들어, 이미지의 시맨틱 해시에 따라 서버에 이미지를 저장하면 같은 서버에는 모두 비슷한 이미지가 저장될 것입니다. 검색 엔진 사용자가 탐색할 이미지를 전달하면, 검색 엔진이 인코더를 사용해 이 이미지의 해시를 계산하고, 이 해시에 해당하는 서버의 모든 이미지를 빠르게 반환할 수 있습니다.
tf.random.set_seed(42)
np.random.seed(42)
hashing_encoder = keras.models.Sequential([
keras.layers.Flatten(input_shape=[28, 28]),
keras.layers.Dense(100, activation="selu"),
keras.layers.GaussianNoise(15.),
keras.layers.Dense(16, activation="sigmoid"),
])
hashing_decoder = keras.models.Sequential([
keras.layers.Dense(100, activation="selu", input_shape=[16]),
keras.layers.Dense(28 * 28, activation="sigmoid"),
keras.layers.Reshape([28, 28])
])
hashing_ae = keras.models.Sequential([hashing_encoder, hashing_decoder])
hashing_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.Nadam(),
metrics=[rounded_accuracy])
history = hashing_ae.fit(X_train, X_train, epochs=10,
validation_data=(X_valid, X_valid))Epoch 1/10
1719/1719 [==============================] - 7s 4ms/step - loss: 0.4066 - rounded_accuracy: 0.8160 - val_loss: 0.3898 - val_rounded_accuracy: 0.8263
Epoch 2/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3688 - rounded_accuracy: 0.8495 - val_loss: 0.3764 - val_rounded_accuracy: 0.8361
Epoch 3/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3585 - rounded_accuracy: 0.8589 - val_loss: 0.3664 - val_rounded_accuracy: 0.8499
Epoch 4/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3530 - rounded_accuracy: 0.8641 - val_loss: 0.3624 - val_rounded_accuracy: 0.8523
Epoch 5/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3497 - rounded_accuracy: 0.8677 - val_loss: 0.3535 - val_rounded_accuracy: 0.8652
Epoch 6/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3472 - rounded_accuracy: 0.8698 - val_loss: 0.3519 - val_rounded_accuracy: 0.8645
Epoch 7/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3447 - rounded_accuracy: 0.8721 - val_loss: 0.3493 - val_rounded_accuracy: 0.8698
Epoch 8/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3432 - rounded_accuracy: 0.8744 - val_loss: 0.3460 - val_rounded_accuracy: 0.8729
Epoch 9/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3415 - rounded_accuracy: 0.8763 - val_loss: 0.3429 - val_rounded_accuracy: 0.8752
Epoch 10/10
1719/1719 [==============================] - 6s 4ms/step - loss: 0.3404 - rounded_accuracy: 0.8776 - val_loss: 0.3433 - val_rounded_accuracy: 0.8778오토인코더는 정보를 매우 크게 (16비트로!) 압축하기 때문에 손실이 많지만 이미지를 재구성하려는 것이 아니라 시맨틱 해시를 생성하는 것이 목적이므로 괜찮습니다:
show_reconstructions(hashing_ae)
plt.show()
출력은 0과 1에 매우 가깝습니다(왼쪽 그래프):
plot_activations_histogram(hashing_encoder)
plt.show()
검증 세트에 있는 처음 몇 개의 이미지에 대한 해시를 확인해 보죠:
hashes = np.round(hashing_encoder.predict(X_valid)).astype(np.int32)
hashes *= np.array([[2**bit for bit in range(16)]])
hashes = hashes.sum(axis=1)
for h in hashes[:5]:
print("{:016b}".format(h))
print("...")0000100101011011
0000100100111011
0100100100011011
0001100101001010
0001000100111100
...이제 검증 세트에 있는 이미지 중 가장 많이 등장하는 해시를 찾아 보죠. 해시마다 몇 개의 샘플을 출력합니다. 다음 이미지에서 한 행에 있는 이미지는 모두 같은 해시를 가집니다:
from collections import Counter
n_hashes = 10
n_images = 8
top_hashes = Counter(hashes).most_common(n_hashes)
plt.figure(figsize=(n_images, n_hashes))
for hash_index, (image_hash, hash_count) in enumerate(top_hashes):
indices = (hashes == image_hash)
for index, image in enumerate(X_valid[indices][:n_images]):
plt.subplot(n_hashes, n_images, hash_index * n_images + index + 1)
plt.imshow(image, cmap="binary")
plt.axis("off")
연습문제 해답
1. to 8.
부록 A 참조
9.
연습문제: 잡음 제거 오토인코더를 사용해 이미지 분류기를 사전훈련해보세요. (간단하게) MNIST를 사용하거나 도전적인 문제를 원한다면 CIFAR10 같은 좀 더 복잡한 이미지 데이터셋을 사용할 수 있습니다. 어떤 데이터셋을 사용하던지 다음 단계를 따르세요.
- 데이터셋을 훈련 세트와 테스트 세트로 나눕니다. 전체 훈련 세트에서 심층 잡음 제거 오토인코더를 훈련합니다.
- 이미지가 잘 재구성되는 지 확인하세요. 코딩 층의 각 뉴런을 가장 크게 활성화하는 이미지를 시각화해보세요.
- 이 오토인코더의 아래 층을 재사용해 분류 DNN을 만드세요. 훈련 세트에서 이미지 500개만 사용해 훈련합니다. 사전훈련을 사용하는 것이 더 나은가요? 사용하지 않는 것이 더 나은가요?
[X_train, y_train], [X_test, y_test] = keras.datasets.cifar10.load_data()
X_train = X_train / 255
X_test = X_test / 255Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
170500096/170498071 [==============================] - 3s 0us/step
170508288/170498071 [==============================] - 3s 0us/steptf.random.set_seed(42)
np.random.seed(42)
denoising_encoder = keras.models.Sequential([
keras.layers.GaussianNoise(0.1, input_shape=[32, 32, 3]),
keras.layers.Conv2D(32, kernel_size=3, padding="same", activation="relu"),
keras.layers.MaxPool2D(),
keras.layers.Flatten(),
keras.layers.Dense(512, activation="relu"),
])denoising_encoder.summary()Model: "sequential_40"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
gaussian_noise_3 (GaussianNo (None, 32, 32, 3) 0
_________________________________________________________________
conv2d_5 (Conv2D) (None, 32, 32, 32) 896
_________________________________________________________________
max_pooling2d_3 (MaxPooling2 (None, 16, 16, 32) 0
_________________________________________________________________
flatten_11 (Flatten) (None, 8192) 0
_________________________________________________________________
dense_46 (Dense) (None, 512) 4194816
=================================================================
Total params: 4,195,712
Trainable params: 4,195,712
Non-trainable params: 0
_________________________________________________________________denoising_decoder = keras.models.Sequential([
keras.layers.Dense(16 * 16 * 32, activation="relu", input_shape=[512]),
keras.layers.Reshape([16, 16, 32]),
keras.layers.Conv2DTranspose(filters=3, kernel_size=3, strides=2,
padding="same", activation="sigmoid")
])denoising_decoder.summary()Model: "sequential_41"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_47 (Dense) (None, 8192) 4202496
_________________________________________________________________
reshape_13 (Reshape) (None, 16, 16, 32) 0
_________________________________________________________________
conv2d_transpose_5 (Conv2DTr (None, 32, 32, 3) 867
=================================================================
Total params: 4,203,363
Trainable params: 4,203,363
Non-trainable params: 0
_________________________________________________________________denoising_ae = keras.models.Sequential([denoising_encoder, denoising_decoder])
denoising_ae.compile(loss="binary_crossentropy", optimizer=keras.optimizers.Nadam(),
metrics=["mse"])
history = denoising_ae.fit(X_train, X_train, epochs=10,
validation_data=(X_test, X_test))Epoch 1/10
1563/1563 [==============================] - 18s 11ms/step - loss: 0.5934 - mse: 0.0186 - val_loss: 0.6037 - val_mse: 0.0217
Epoch 2/10
1563/1563 [==============================] - 16s 10ms/step - loss: 0.5726 - mse: 0.0100 - val_loss: 0.5762 - val_mse: 0.0109
Epoch 3/10
1563/1563 [==============================] - 16s 10ms/step - loss: 0.5676 - mse: 0.0080 - val_loss: 0.5725 - val_mse: 0.0095
Epoch 4/10
1563/1563 [==============================] - 16s 10ms/step - loss: 0.5655 - mse: 0.0072 - val_loss: 0.5724 - val_mse: 0.0094
Epoch 5/10
1563/1563 [==============================] - 16s 10ms/step - loss: 0.5640 - mse: 0.0067 - val_loss: 0.5678 - val_mse: 0.0075
Epoch 6/10
1563/1563 [==============================] - 16s 10ms/step - loss: 0.5631 - mse: 0.0063 - val_loss: 0.5667 - val_mse: 0.0072
Epoch 7/10
1563/1563 [==============================] - 16s 10ms/step - loss: 0.5624 - mse: 0.0060 - val_loss: 0.5655 - val_mse: 0.0067
Epoch 8/10
1563/1563 [==============================] - 17s 11ms/step - loss: 0.5620 - mse: 0.0059 - val_loss: 0.5652 - val_mse: 0.0066
Epoch 9/10
1563/1563 [==============================] - 16s 10ms/step - loss: 0.5616 - mse: 0.0058 - val_loss: 0.5647 - val_mse: 0.0064
Epoch 10/10
1563/1563 [==============================] - 16s 10ms/step - loss: 0.5613 - mse: 0.0056 - val_loss: 0.5639 - val_mse: 0.0061n_images = 5
new_images = X_test[:n_images]
new_images_noisy = new_images + np.random.randn(n_images, 32, 32, 3) * 0.1
new_images_denoised = denoising_ae.predict(new_images_noisy)
plt.figure(figsize=(6, n_images * 2))
for index in range(n_images):
plt.subplot(n_images, 3, index * 3 + 1)
plt.imshow(new_images[index])
plt.axis('off')
if index == 0:
plt.title("Original")
plt.subplot(n_images, 3, index * 3 + 2)
plt.imshow(np.clip(new_images_noisy[index], 0., 1.))
plt.axis('off')
if index == 0:
plt.title("Noisy")
plt.subplot(n_images, 3, index * 3 + 3)
plt.imshow(new_images_denoised[index])
plt.axis('off')
if index == 0:
plt.title("Denoised")
plt.show()
10.
연습문제: 이미지 데이터셋을 하나 선택해 변이형 오토인코더를 훈련하고 이미지를 생성해보세요. 또는 관심있는 레이블이 없는 데이터셋을 찾아서 새로운 샘플을 생성할 수 있는지 확인해 보세요.
11.
연습문제: 이미지 데이터셋을 처리하는 DCGAN을 훈련하고 이를 사용해 이미지를 생성해보세요. 경험 재생을 추가하고 도움이 되는지 확인하세요. 생성된 클래스를 제어할 수 있는 조건 GAN으로 바꾸어 시도해보세요.